'''
Company: TWL
Author: xue jian
Email: xuejian@kanzhun.com
Date: 2020-10-14 21:12:41
'''
'''
type-value与具体type的对应关系：
INT(1, "int"),
LONG(2, "long"),
FLOAT(3, "float"),
DOUBLE(4, "double"),
INT_VECTOR(5, "int-vector"),
FLOAT_VECTOR(6, "float-vector"),
LONG_VECTOR(7, "long-vector"),
DOUBLE_VECTOR(8, "double-vector"),
STRING(9,"string"),
WCMP(13, "L1-Code-W2V序列化"),
STRING_STRING_VALUE(14, "kv[key=string,value=string]"),
STRING_FLOAT_VALUE(15, "kv[key=string,value=float]"),
INT_FLOAT_VALUE(16, "kv[key=int,value=float]");
'''
value_type2method = {1: 'int2fid', 2: 'long2fid', 3: 'float2fid', 4: 'float2fid', 7:'longarray2fid', 9:'string2fid'}

'''
dimension与具体维度的对应关系
BOSS(1, "Boss(老板)"),
GEEK(2, "Geek(牛人)"),
JOB(3, "Job(职位)"),
EXPECT(4, "Expect(期望)"),
BOSS_GEEK(5, "BossGeek(两两)"),
JOB_EXPECT(6, "JobExpect(两两)"),
COMPANY(7, "Company(公司)"),
SALE(8, "Sale(销售)"),
SALE_COMPANY(10, "SaleCompany(两两)"),
PUSH_TEXT(11, "PushText(Push)"),
USER(13, "User(用户)"),
GEEK_JOB(14, "GeekJob(两两)"),
CITY(15, "City(城市)"),
CONTEXT(16, "Context(上下文)"),
'''

dimension2true = {1:'Boss', 2:'Geek', 3: "Job(职位)", 4: "Expect"}

import json
import os
import sys
import json
sys.path.append(os.path.abspath(os.path.join(os.path.dirname('__file__'), os.path.pardir)))

import thrift
from thrift.transport import TSocket
from thrift.transport import TTransport
from thrift.protocol import TBinaryProtocol
from hulk_proxy.HulkProxyServ import Client
from scan_data.ttypes import ScanData

__HOST = "172.21.32.54"
__PORT = 10088

import pymysql
import json
final = []
conn = pymysql.connect(
    host = "172.19.8.66",
    port = 3306,
    user = "algorithm",
    password = "S4!4Eywru@aH",
    # database = "rcd_phoenix_admin",
    database = "rcd_kraken",
    charset='utf8'
    )

cursor = conn.cursor()

def get_feature_meta_sql():
    import pymysql
    import json
    final = []
    conn = pymysql.connect(
        host = "172.19.8.66",
        port = 3306,
        user = "algorithm",
        password = "S4!4Eywru@aH",
        # database = "rcd_phoenix_admin",
        database = "rcd_kraken",
        charset='utf8'
        )

    cursor = conn.cursor()
    select = ['code', 'feature_name', 'dimension', 'value_type', 'default_value']
    sql = 'select ' + ','.join(select) + ' from feature_meta'
    cursor.execute(sql)
    result = list(cursor.fetchall())
    # print(result[0])
    re = {}
    for line in result:
        name = line[1]
        if name not in re:
            re[name] = {}
        for i, v in enumerate(select):
            re[name][v] = line[i]
    return re



add_fea = ['job_pdet_rate_14d', 'b2g_apply_status_recent10_realtime', 'geek_workyears', 'b2g_position_recent10_realtime', 'job_paddf_success_rate_7d', 'geek_paddf_rate_14d', 'el', 'geek_pdet_rate_14d', 'geek_paddf_success_rate_7d', 'jh', 'job_paddf_rate_14d', 'exp_pas_addf_num_24h', 'b2g_salary_recent10_realtime', 'boss_position', 'geek_apply_status', 'geek_school_type1', 'job_workyears', 'b2g_revage_recent10_realtime', 'five_job_addf_degree_recent20', 'geek_rev_age', 'b2g_school_type1_recent10_realtime', 'exp_id', 'jl', 'job_paddfchat_times_2d', 'five_job_addf_workyears_recent20', 'geek_fresh_graduate', 'geek_rev_work_year', 'job_id', 'latest_company_kwid', 'geek_major', 'geek_cmp_level', 'geek_school_level', 'geek_degree_new', 'job_paddf_pchat_rate_7d', 'geek_psuccess_rate_7d', 'geek_pre_salary', 'geek_cmp_pas_addf_rate', 'job_degree', 'geek_sch_pas_addf_rate', 'geek_latest_work_position_id', 'geek_degree', 'geek_district_code', 'b2g_school_type2_recent10_realtime', 'geek_gender', 'geek_paddf_pchat_rate_7d', 'exp_procwilling_rate_1d3', 'b2g_cmp_level_recent10_realtime', 'b2g_school_level_recent10_realtime', 'b2g_gender_recent10_realtime', 'geek_school_type2', 'b2g_paspos_recent10_realtime', 'geek_position', 'job_pas_addf_num_24h', 'exp_paddfchat_rate_3d']
def get_feature_meta_hulk():
    try:
        tsocket = TSocket.TSocket(__HOST, __PORT)
        transport = TTransport.TFramedTransport(tsocket)
        protocol = TBinaryProtocol.TBinaryProtocol(transport)
        client = Client(protocol)

        transport.open()


        feature_code = client.hget('feature_meta', 'feature_code')
        feature_code = json.loads(feature_code)

        feature_slot = client.hget('feature_meta', 'feature_slot')
        feature_slot = json.loads(feature_slot)

        cut_points = client.hget('feature_meta', 'cut_points')
        cut_points = json.loads(cut_points)

    except thrift.TException as ex:
        print(ex.message)
    return feature_slot, feature_code, cut_points


# for fea in add_fea:
#     if fea not in fea_code_map:
#         print(fea)
#     if fea in feature_slot:
#         print(fea)


def gen_default(f_meta):
    if f_meta['value_type']==1:
        return int(f_meta['default_value'])
    elif f_meta['value_type']==2:
        return int(f_meta['default_value'])
    elif f_meta['value_type']==3 or f_meta['value_type']==4:
        return float(f_meta['default_value'])
    elif f_meta['value_type']==7:
        return [int(f_meta['default_value'])]*10
    elif f_meta['value_type']==9:
        return f_meta['default_value']

def int2str_feature_code():
    _, feature_code, _ = get_feature_meta_hulk()
    codes = ['boss_fea_codes', 'boss_fea_codes_recall', 'geek_fea_codes', 'geek_fea_codes_recall']
    for code_type in codes:
        for i in range(len(feature_code[code_type])):
            feature_code[code_type][i] = str(feature_code[code_type][i])
    # print(feature_code)
    with open('feature/feature_code.json', 'w') as f:
        json.dump(feature_code, f)


def add_feature_function():
    fea_meta = get_feature_meta_sql()
    feature_slot, feature_code, cut_points = get_feature_meta_hulk()
    # cnt = 0
    # for k,v in feature_slot.items():
    #     if k in fea_meta:
    #         method=value_type2method[fea_meta[k]['value_type']]
    #         if method != v['method']:
    #             print(k, 'method', v['method'], ', but the feature meta its method is',method)
    #             cnt+=1
    # print(cnt)
    # print((feature_slot))
    int_set = set(range(1, 168))
    for k, v in feature_slot.items():
        int_set.discard(v['slot'])
    print(len(int_set))
    int_set = list(int_set)
    add_feature = []
    for fea in add_fea:
        if fea not in feature_slot:
            add_feature.append(fea)

    print(len(add_feature))

    
    ind = 0
    need_cut_points = []
    for fea in add_feature:
        # add to the feature_slot
        if fea not in feature_slot:
            # print(fea_meta[fea])
            if fea_meta[fea]['value_type'] == 4 or fea_meta[fea]['value_type'] == 3:
                print(fea_meta[fea])
                need_cut_points.append(fea)
                # continue
            feature_slot[fea] = {}
            feature_slot[fea]['slot'] = int_set[ind]
            ind+=1
            feature_slot[fea]['method'] = value_type2method[fea_meta[fea]['value_type']]
            feature_slot[fea]['deps'] = [fea]

            # add to the feature_code
            code = fea_meta[fea]['code']
            dimension = fea_meta[fea]['dimension']
            name = fea_meta[fea]['feature_name']
            if dimension==1 or dimension==3:
                feature_code['boss_fea_codes'].append(str(code))
                feature_code['boss_fea_codes_recall'].append(str(code))
                feature_code['boss_features'][code] = name
                feature_code['boss_fea_default'][code] = gen_default(fea_meta[fea])
            elif dimension==2 or dimension==4:
                feature_code['geek_fea_codes'].append(str(code))
                feature_code['geek_fea_codes_recall'].append(str(code))
                feature_code['geek_features'][code] = name
                feature_code['geek_fea_default'][code] = gen_default(fea_meta[fea])


    # add to the cut_points
    
    with open('feature/cut_boundary.json') as f:
        cut_default = json.load(f)
    for fea in need_cut_points:
        # print(fea)
        # print(cut_default[fea])
        cut_points[str(feature_slot[fea]['slot'])] = cut_default[fea]
    # for fea

    with open('feature/feature_slot.json', 'w') as f:
        json.dump(feature_slot, f)
    
    with open('feature/feature_code.json', 'w') as f:
        json.dump(feature_code, f)

    with open('feature/cut_points.json', 'w') as f:
        json.dump(cut_points, f)

def correct_feature_slot_str2float():
    feature_slot, _, cut_points = get_feature_meta_hulk()
    with open('feature/cut_boundary.json') as f:
        cut_default = json.load(f)
    for k, v in cut_default.items():
        if k not in feature_slot:
            continue
        cut_points[str(feature_slot[k]['slot'])] = v
    with open('feature/cut_points.json', 'w') as f:
        json.dump(cut_points, f)
    # print(cut_points)
    for k, v in feature_slot.items():
        if str(v['slot']) in cut_points:
            v['method'] = 'float2fid'
    # print(feature_slot)
    with open('feature/feature_slot.json', 'w') as f:
        json.dump(feature_slot, f)

if __name__ == "__main__":
    # add_feature_function()
    # int2str_feature_code()
    correct_feature_slot_str2float()
